package com.surfilter.gsxt.util;

import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.ParseException;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import org.json.JSONObject;
import org.jsoup.Connection;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.*;
import org.jsoup.select.Elements;
import org.openqa.selenium.*;
import org.openqa.selenium.phantomjs.PhantomJSDriver;
import org.openqa.selenium.phantomjs.PhantomJSDriverService;
import org.openqa.selenium.remote.DesiredCapabilities;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.X509TrustManager;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.security.MessageDigest;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;

/**
 * 实现HTTP POST
 */
public final class HttpUtil {
	
	private static final Logger logger = LoggerFactory.getLogger(HttpUtil.class);

	private static ResourceBundle bundle = ResourceBundle.getBundle("collect-config");
	
	private static final String phantomjsPath = bundle.getString("phantomjs_driver_path");
	
	private static final String crawlerSyncUrl = bundle.getString("crawler_sync_url");

	private static X509TrustManager tm = new X509TrustManager() {

		public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
			// TODO Auto-generated method stub
			
		}

		public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
			// TODO Auto-generated method stub
			
		}

		public X509Certificate[] getAcceptedIssuers() {
			// TODO Auto-generated method stub
			return null;
		}
		
	};

	private static SSLContext sslContext = null;
	
	private static Set<String> topd = new HashSet<String>();
	
	public static Set<String> titleBlacklist = new HashSet<String>();
	
	private static Set<String> excludeTags = new HashSet<String>();

	static {
		try {
			sslContext = SSLContext.getInstance("SSL", "SunJSSE");
			sslContext.init(null, new X509TrustManager[] { tm }, new java.security.SecureRandom());
		} catch (Exception e) {
			e.printStackTrace();
		}
		
		//网页标题黑名单
		titleBlacklist.add("阿里云-备案-阻断页-PC");
		titleBlacklist.add("TestPage");
		titleBlacklist.add("Apache Tomcat");
		titleBlacklist.add("阿里云万网虚机过期页面");
		titleBlacklist.add("访问出错");
		titleBlacklist.add("在从服务器接收结果时发生传输级错误");
		titleBlacklist.add("页面不存在_百度搜索");
		titleBlacklist.add("Default Web Site Page");
		titleBlacklist.add("bbs.esudai.com - Database Error");
		titleBlacklist.add("Forbidden 禁止访问");
		titleBlacklist.add("阿里云万网虚机手工关停页面");
		titleBlacklist.add("IIS7");
		
		
		// 双后缀域名先过滤
		topd.add(".hb.cn");
		topd.add(".net.cn");
		topd.add(".com.cn");
		topd.add(".yn.cn");
		topd.add(".ha.cn");
		topd.add(".he.cn");
		topd.add(".com.au");
		topd.add(".nx.cn");
		topd.add(".sd.cn");
		topd.add(".sn.cn");
		topd.add(".ah.cn");
		topd.add(".bj.cn");
		topd.add(".gx.cn");
		topd.add(".hl.cn");
		topd.add(".gd.cn");
		topd.add(".ac.cn");
		topd.add(".gs.cn");
		topd.add(".xj.cn");
		topd.add(".sc.cn");
		topd.add(".ln.cn");
		topd.add(".hk.cn");
		topd.add(".zj.cn");
		topd.add(".gov.cn");
		topd.add(".edu.cn");
		topd.add(".cn.com");
		topd.add(".hn.cn");
		topd.add(".gz.cn");
		topd.add(".js.cn");
		topd.add(".com.hk");
		topd.add(".sh.cn");
		topd.add(".fj.cn");
		topd.add(".nm.cn");
		topd.add(".net.au");
		topd.add(".mo.cn");
		topd.add(".tj.cn");
		topd.add(".jx.cn");
		topd.add(".jl.cn");
		topd.add(".qh.cn");
		topd.add(".cq.cn");
		topd.add(".org.cn");
		topd.add(".xz.cn");
		topd.add(".hi.cn");
		topd.add(".tw.cn");
		topd.add(".com.tw");
		topd.add(".sx.cn");
		
		excludeTags.add("head");
		excludeTags.add("script");
		excludeTags.add("style");
	}

	// 通用的http传输方法
	public static String post(String url, JSONObject json) throws IOException {
		URL url1 = null;
		BufferedReader reader = null;
		PrintWriter writer = null;
		OutputStreamWriter out = null;
		HttpURLConnection connection = null;
		try {
			url1 = new URL(url);

			connection = (HttpURLConnection) url1.openConnection();
			connection.setConnectTimeout(30000);
			connection.setReadTimeout(30000);
			connection.setRequestMethod("POST");
			connection.setInstanceFollowRedirects(true);

			connection.setDoInput(true);
			connection.setDoOutput(true);
			connection.setRequestProperty("Content-Type", "application/json;charset=UTF-8");
			connection.connect();
			out = new OutputStreamWriter(new BufferedOutputStream(connection.getOutputStream()), "UTF-8");
			out.write(json.toString());
			out.flush();
			out.close();
			System.out.println(connection.getResponseCode());
			if (connection.getResponseCode() >= 400) {
				throw new IllegalStateException("对方返回错误码:" + connection.getResponseCode());
			}
			reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8"));
			StringBuffer sb = new StringBuffer();
			String line;
			while ((line = reader.readLine()) != null) {
				sb.append(line);
			}
			return sb.toString();
		} finally {
			if (reader != null) {
				try {
					reader.close();
				} catch (IOException e1) {
				}
			}
			if (writer != null) {
				writer.close();
			}
			if (out != null) {
				out.close();
			}
			if (connection != null)
				connection.disconnect();
		}
	}
	
	// 通用的http传输方法
	public static String post(String url, String params, Map<String, String> headers) throws IOException {
		URL url1 = null;
		BufferedReader reader = null;
		PrintWriter writer = null;
		OutputStreamWriter out = null;
		HttpURLConnection connection = null;
		try {
			url1 = new URL(url);

			connection = (HttpURLConnection) url1.openConnection();
			connection.setConnectTimeout(30000);
			connection.setReadTimeout(30000);
			connection.setRequestMethod("POST");
			connection.setInstanceFollowRedirects(true);

			connection.setDoInput(true);
			connection.setDoOutput(true);
			connection.setRequestProperty("Content-Type", "application/json;charset=UTF-8");
			if(headers != null){
				for(String key : headers.keySet()){
					connection.setRequestProperty(key, headers.get(key));
				}
			}
			connection.connect();
			out = new OutputStreamWriter(new BufferedOutputStream(connection.getOutputStream()), "UTF-8");
			out.write(params);
			out.flush();
			out.close();
			System.out.println(connection.getResponseCode());
//			if (connection.getResponseCode() >= 400) {
//				throw new IllegalStateException("对方返回错误码:" + connection.getResponseCode());
//			}
			reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8"));
			StringBuffer sb = new StringBuffer();
			String line;
			while ((line = reader.readLine()) != null) {
				sb.append(line);
			}
			return sb.toString();
		} finally {
			if (reader != null) {
				try {
					reader.close();
				} catch (IOException e1) {
				}
			}
			if (writer != null) {
				writer.close();
			}
			if (out != null) {
				out.close();
			}
			if (connection != null)
				connection.disconnect();
		}
	}

	public static byte[] getPostByte(String url, JSONObject json) throws IOException {
		URL url1 = null;
		BufferedInputStream bufferedInputStream = null;
		OutputStreamWriter out = null;
		HttpURLConnection connection = null;

		url1 = new URL(url);

		connection = (HttpURLConnection) url1.openConnection();
		connection.setConnectTimeout(10000);
		connection.setReadTimeout(10000);
		connection.setRequestMethod("POST");
		connection.setInstanceFollowRedirects(true);

		connection.setDoInput(true);
		connection.setDoOutput(true);
		connection.setRequestProperty("Content-Type", "application/json;charset=UTF-8");
		connection.connect();
		out = new OutputStreamWriter(new BufferedOutputStream(connection.getOutputStream()), "UTF-8");
		out.write(json.toString());
		out.flush();
		out.close();
		System.out.println(connection.getResponseCode());
		if (connection.getResponseCode() >= 400) {
			throw new IllegalStateException("对方返回错误码:" + connection.getResponseCode());
		}

		bufferedInputStream = new BufferedInputStream(connection.getInputStream());
		ByteArrayOutputStream bos = new ByteArrayOutputStream(bufferedInputStream.available());

		try {

			int buf_size = 1024;
			byte[] buffer = new byte[buf_size];
			int len = 0;
			while (-1 != (len = bufferedInputStream.read(buffer, 0, buf_size))) {
				bos.write(buffer, 0, len);
			}
			bos.flush();
			bos.toByteArray();

			return bos.toByteArray();
		} finally {
			if (bufferedInputStream != null) {
				try {
					bufferedInputStream.close();
				} catch (IOException e1) {
				}
			}
			if (connection != null) {
				connection.disconnect();
			}
		}
	}

	// 通用的http传输方法
	public static String get(String url, JSONObject json) throws IOException {
		URL url1 = null;
		BufferedReader reader = null;
		PrintWriter writer = null;
		OutputStreamWriter out = null;
		HttpURLConnection connection = null;
		try {
			url1 = new URL(url);

			connection = (HttpURLConnection) url1.openConnection();
			connection.setConnectTimeout(10000);
			connection.setReadTimeout(10000);
			connection.setRequestMethod("GET");
			connection.setInstanceFollowRedirects(true);

			connection.setDoInput(true);
			connection.setDoOutput(true);
			connection.setRequestProperty("Content-Type", "application/json;charset=UTF-8");
			connection.connect();
			out = new OutputStreamWriter(new BufferedOutputStream(connection.getOutputStream()), "UTF-8");
			if (json != null)
				out.write(json.toString());
			out.flush();
			out.close();
			System.out.println(connection.getResponseCode());
			if (connection.getResponseCode() >= 400) {
				throw new IllegalStateException("对方返回错误码:" + connection.getResponseCode());
			}
			String encoding = null;
			encoding = connection.getContentEncoding();

			reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), encoding));
			StringBuffer sb = new StringBuffer();
			String line;
			while ((line = reader.readLine()) != null) {
				sb.append(line);
			}
			return sb.toString();
		} finally {
			if (reader != null) {
				try {
					reader.close();
				} catch (IOException e1) {
				}
			}
			if (writer != null) {
				writer.close();
			}
			if (out != null) {
				out.close();
			}
			if (connection != null)
				connection.disconnect();
		}
	}
	
	// 通用的http传输方法
	public static String get(String url, Map<String, String> headers) throws IOException {
		URL url1 = null;
		BufferedReader reader = null;
		PrintWriter writer = null;
		OutputStreamWriter out = null;
		HttpURLConnection connection = null;
		try {
			url1 = new URL(url);

			connection = (HttpURLConnection) url1.openConnection();
			connection.setConnectTimeout(10000);
			connection.setReadTimeout(10000);
			connection.setRequestMethod("GET");
			connection.setDoOutput(true);
			if(headers != null){
				for(String key : headers.keySet()){
					connection.setRequestProperty(key, headers.get(key));
				}
			}
			connection.connect();
			out = new OutputStreamWriter(new BufferedOutputStream(connection.getOutputStream()), "UTF-8");
			out.flush();
			out.close();
			System.out.println(connection.getResponseCode());
			if (connection.getResponseCode() >= 400) {
				throw new IllegalStateException("对方返回错误码:" + connection.getResponseCode());
			}
			String encoding = null;
			encoding = connection.getContentEncoding();

			reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), encoding));
			StringBuffer sb = new StringBuffer();
			String line;
			while ((line = reader.readLine()) != null) {
				sb.append(line);
			}
			return sb.toString();
		} finally {
			if (reader != null) {
				try {
					reader.close();
				} catch (IOException e1) {
				}
			}
			if (writer != null) {
				writer.close();
			}
			if (out != null) {
				out.close();
			}
			if (connection != null)
				connection.disconnect();
		}
	}
	
	public static String getHtmlTitle(String html){
		String result = null;
		try{
			if(html == null)return result;
			Pattern patt = Pattern.compile("<title[^<^>]*>([^<^>]*)</title>");
			Matcher match = patt.matcher(html);
			String retitle = null;
			while(match.find()){
				retitle = match.group();
				if(retitle.length() > 0)break;
			}
			if(retitle == null)return result;
			retitle = retitle.trim();
			result = retitle.replaceAll("</?title[^<^>]*>", "");
		}catch(Exception e){
			logger.error("获取HTML Title异常", e);
		}
		return result;
	}
	
	public static String clearHtmlNotes(String html){
		String regex = "<!--[\\w\\W]*?-->";
		html = html.replaceAll(regex, "");
		return html;
	}
	
	public static String getHtmlDescription(String html){
		String regex1 = "<meta\\s*(http-equiv|name)\\s*=\\s*\"(\\s*[kK][eE][yY][wW][oO][rR][dD][sS]|description|[kK][eE][yY][wW][oO][rR][dD])\\s*\"\\s*content\\s*=\\s*\"[^<^>]*\"\\s*>(([^<^>]*)</meta>)?";
		String regex2 = "content\\s*=\\s*\"[^<^>]*\"";
		String regex3 = "\".*\"";
		String result = "";
		result = StringParseUtils.parseString(html, regex1, true);
		result = StringParseUtils.parseString(result, regex2, true);
		result = StringParseUtils.parseString(result, regex3, true);
		result = result.replaceAll("\"", "");
		return result;
	}
	
	public static String getHtmlBodyText(String html){
		String result = "";
		String regex1 = "<\\s*body\\s*[^<^>]*>([\\s\\S]*)</\\s*body\\s*[^<^>]*>";
		String regex2 = "<script[^<^>]*/?>[^<^>]*(</script[^<^>]*>)?";
		String regex3 = "<[^<^>]*>(</[^<^>]*>)?";
		result = StringParseUtils.parseString(html, regex1, true);
		result = result.replaceAll(regex2, "");
		result = result.replaceAll(regex3, "");
		return result;
	}
	
	public static List<String> getHtmlBodyTexts(String html){
		List<String> result = new ArrayList<String>();
		String resultStr = "";
		String regex2 = "<script[^<^>]*/?>[^<^>]*(</script[^<^>]*>)?";
		String regex3 = "<[^<^>]*>(</[^<^>]*>)?";
		String regex4 = "<[^<^>]*>[^<^>]*</[^<^>]*>";
		resultStr = getHtmlBody(html);
		resultStr = resultStr.replaceAll(regex2, "");
		List<String> resultB = StringParseUtils.parseStringL(resultStr, regex4, true);
		if(resultB != null){
			for(String str : resultB){
				result.add(str.replaceAll(regex3, ""));
			}
		}
		return result;
	}
	
	public static void cleanUpHidinTags(Node ele) {
		
		String disNone = null;
		if (ele == null)
			return;
		
		if(ele.hasAttr("style")){
			disNone = StringParseUtils.parseString(ele.attr("style"), "display\\s*:\\s*none", false);
		}
		
		if (ele.hasAttr("hidden") || (disNone != null && !disNone.trim().equalsIgnoreCase(""))
				|| (ele.hasAttr("type") && ele.attr("type").contains("hidden"))) {
			ele.remove();
			return;
		}else{
			if(ele instanceof Comment){
				ele.remove();
			}else if(ele instanceof Element){
				Element subele = (Element)ele;
				String tagName = subele.tagName();
				if(excludeTags.contains(tagName)){
					subele.remove();
					return;
				}
				if(subele.children() == null || subele.children().size() == 0)
					return;
				Elements eles = subele.children();
				for(int i = 0 ; i < eles.size() ; i ++){
					cleanUpHidinTags(eles.get(i));
				}
			}else if(ele instanceof TextNode){
				System.out.println("TextNode :" + ele.nodeName());
			}else if(ele instanceof DataNode){
				System.out.println("DataNode :" + ele.nodeName());
			}else if(ele instanceof DocumentType){
				System.out.println("DocumentType :" + ele.nodeName());
			}else if(ele instanceof XmlDeclaration){
				System.out.println("XmlDeclaration :" + ele.nodeName());
			}
		}
	}
	
	public static String clearHtmlHideTags(String html){
		
		
		
		return html;
	}
	
	public static String clearScript(String html){
		String result = html;
		if(!StringUtils.isEmpty(html) && !StringUtils.isBlank(html.trim())){
			String regex2 = "<script[^<^>]*/?>[^<^>]*(</script[^<^>]*>)?";
			result = result.replaceAll(regex2, "");
		}
		return result;
	}
	
	public static String getHtmlBody(String html){
		String body = null;
		String regex1 = "<\\s*body\\s*[^<^>]*>([\\s\\S]*)</\\s*body\\s*[^<^>]*>";
		if(!StringUtils.isEmpty(html) && !StringUtils.isBlank(html.trim())){
			body = StringParseUtils.parseString(html, regex1, true);
		}
		return body;
	}
	
	public static int getHtmlLabelNum(String html){
		int result = -1;
		try{
			result = StringParseUtils.parseLength(html, "<[^<^>]*>(</[^<^>]*>)?");
		}catch(Exception e){
			logger.error("获取Html标签个数异常,", e);
		}
		return result;
	}
	
	public static int visitException(List<String> htmls){
		int result = 3;
		if(htmls != null && !htmls.isEmpty()){
			int htmlLabelNum = 0;
			int htmlLength = 0;
			for(String html : htmls){
				String title = getHtmlTitle(html);
				if(title != null && !title.trim().equalsIgnoreCase("") &&isExsit(title)){
					return 1;
				}
				html = getHtmlBody(html);
				html = clearScript(html);
				htmlLabelNum += getHtmlLabelNum(html);
				htmlLength += html.length();
			}
			if(htmlLabelNum < 50 && htmlLength < 5000)result = 1;
			else result = 2;
		}
		return result;
	}
	
	public static boolean isExsit(String str){
		boolean target = false;
		if(titleBlacklist.contains(str))return true;
		for(String titleBlack : titleBlacklist){
			if(titleBlack.contains(str)){
				target = true;
				break;
			}
		}
		return target;
	}
	
	
	public static String get(String url, String ecoding) throws IOException {
		URL url1 = null;
		BufferedReader reader = null;
		PrintWriter writer = null;
		OutputStreamWriter out = null;
		HttpURLConnection connection = null;
		try {
			url1 = new URL(url);

			connection = (HttpURLConnection) url1.openConnection();
			connection.setConnectTimeout(10000);
			connection.setReadTimeout(10000);
			connection.setRequestMethod("GET");
			connection.setInstanceFollowRedirects(true);
			connection.setDoInput(true);
			connection.setDoOutput(true);
//			tnet=111.205.62.10; TYCID=b432a0ffe4bf496a9b30e9bc3fed1b18; _pk_ref.1.e431=%5B%22%22%2C%22%22%2C1466659898%2C%22https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3DBd5vW0eWsdqWK12oURfg4f6gc15YJt33AszbXecJgAUWWZqPeOSh-42Jh9IpIb59%26wd%3D%26eqid%3Deaad6d450000eb2300000002576b5227%22%5D; _pk_id.1.e431=5044394bcf92ce92.1466656724.2.1466660117.1466659898.; _pk_ses.1.e431=*; Hm_lvt_e92c8d65d92d534b0fc290df538b4758=1466659898,1466660117; Hm_lpvt_e92c8d65d92d534b0fc290df538b4758=1466660117; token=239780cb749442879ed46eae05274a64; _utm=4b6327a3a8074953a0182b2205d3588f
//			connection.setRequestProperty("Content-Type", "application/json;charset=UTF-8");
			connection.setRequestProperty("Accept", "application/json, text/plain, */*");
			connection.setRequestProperty("Accept-Encoding", "gzip,deflate,sdch");
			connection.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.8");
			connection.setRequestProperty("CheckError", "check");
			connection.setRequestProperty("Connection", "keep-alive");
			connection.setRequestProperty("Pragma", "no-cache");
			connection.setRequestProperty("Upgrade-Insecure-Requests", "1");
			connection.setRequestProperty("User-Agent",
					"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36");
			connection.setRequestProperty("Cookie", "tnet=111.205.62.10; TYCID=b432a0ffe4bf496a9b30e9bc3fed1b18; _pk_ref.1.e431=%5B%22%22%2C%22%22%2C1466659898%2C%22https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3DBd5vW0eWsdqWK12oURfg4f6gc15YJt33AszbXecJgAUWWZqPeOSh-42Jh9IpIb59%26wd%3D%26eqid%3Deaad6d450000eb2300000002576b5227%22%5D; _pk_id.1.e431=5044394bcf92ce92.1466656724.2.1466660117.1466659898.; _pk_ses.1.e431=*; Hm_lvt_e92c8d65d92d534b0fc290df538b4758=1466659898,1466660117; Hm_lpvt_e92c8d65d92d534b0fc290df538b4758=1466660117; token=239780cb749442879ed46eae05274a64; _utm=4b6327a3a8074953a0182b2205d3588f");
			connection.connect();
			if (connection.getResponseCode() >= 400) {
				throw new IllegalStateException("对方返回错误码:" + connection.getResponseCode());
			}
			String encoding = null;
			encoding = connection.getContentEncoding();
			if(encoding == null)encoding = ecoding;
			reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), encoding));
			StringBuffer sb = new StringBuffer();
			String line;
			while ((line = reader.readLine()) != null) {
				sb.append(line);
			}
			return sb.toString();
		} finally {
			if (reader != null) {
				try {
					reader.close();
				} catch (IOException e1) {
				}
			}
			if (writer != null) {
				writer.close();
			}
			if (out != null) {
				out.close();
			}
			if (connection != null)
				connection.disconnect();
		}
	}

	public static String saveUrlToFile(String url, String filepath) throws Exception {
		Connection con = Jsoup.connect(url);
		con.validateTLSCertificates(false);

		con.timeout(30000);
		con.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
		con.header("Accept-Encoding", "gzip, deflate, sdch");
		con.header("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4");
		con.header("Cache-Control", "no-cache");
		con.header("Connection", "keep-alive");
		con.header("Pragma", "no-cache");
		con.header("Upgrade-Insecure-Requests", "1");
		con.header("User-Agent",
				"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36");
		
		Document doc = con.get();
		if (doc != null) {
			String html = doc.html();
			PrintWriter w = new PrintWriter(filepath);
			w.println(html);
			w.flush();
			w.close();
		}
		
		//System.out.println(filepath);
		return filepath;
	}
	
	/**
	 * 通过domain(域名)获取网站Html标签,并将Html标签存入指定文件
	 * 返回正确打开网站的Url
	 */
	public static String saveWebHtmlByDomain(String domain, String file_path){
		String url = null;
		List<String> urls = getUrls(domain);
		for(String rurl : urls){
			Response resp = getResponse(rurl);
			if(resp == null)continue;
			try {
				Document doc = resp.parse();
				if(doc == null)continue;
				String html = doc.html();
				if(html == null || html.trim().equalsIgnoreCase(""))continue;
				
			} catch (Exception e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
				logger.error("打开网站：{}异常", rurl, e);
				continue;
			}
			
		}
		return url;
	}
	
	public static boolean webJump(Document doc){
		boolean target = false;
		
		String metaUrl = null;
		Elements head = doc.getElementsByTag("head");
		Elements meta = doc.select("meta[http-equiv=refresh]");
		if(meta != null && meta.size() > 0){
			metaUrl = meta.get(0).attr("content");
		}else{
			metaUrl = null;
		}
		
		if(metaUrl == null){
			if(head != null && head.size() > 0){
				String values = StringParseUtils.parseString(head.toString(), "\\s*meta\\s*http-equiv\\s*=\\s*\\'?refresh\\'?\\s*content\\s*=\\s*\\'.+\\'", false);
				if(values != null && !values.trim().equalsIgnoreCase("")){
					metaUrl = values;
				}
			}
		}
		if(metaUrl == null){
			Elements scriptEle = doc.getElementsByTag("script");
			Elements iframeEle = doc.getElementsByTag("iframe");
			Elements frameEle = doc.getElementsByTag("frame");
			Element bodyEle = doc.body();
			//判断iframe标签存在否
			if(iframeEle != null && iframeEle.size() > 0 && !target){
				for(int i = 0; i < iframeEle.size(); i++){
					String framesrc = iframeEle.get(i).attr("src");
					if(framesrc != null && !framesrc.trim().equals("")){
						target = true;
						break;
					}
				}
			}
			//判断frame标签存在否
			if(frameEle != null && frameEle.size() > 0 && !target){
				for(int i = 0; i < frameEle.size(); i++){
					String framesrc = frameEle.get(i).attr("src");
					if(framesrc != null && !framesrc.trim().equals("")){
						target = true;
						break;
					}
				}
			}
			
			//判断js跳转
			if(scriptEle != null && scriptEle.size() > 0  && !target){
				String values = StringParseUtils.parseString(scriptEle.toString(), ".*\\.?location(\\.[a-z]*)?\\s*=.*", false);
				if((values != null && !values.trim().equalsIgnoreCase("")) || scriptEle.toString().contains("window.onload")){
					target = true;
				}else{
					target = (scriptEle.toString().contains("XMLHttpRequest") || scriptEle.toString().contains(".submit"));
				}
			}
			//判断body跳转存在不
			if(bodyEle != null && !target){
				String bodyOnload = bodyEle.attr("onload");
				if(bodyOnload != null && !bodyOnload.trim().equalsIgnoreCase(""))target = true;
			}
		}else{
			target = true;
		}
		return target;
	}
	
	public static String saveUrlToFileRHtml(String url, String filepath) throws Exception {
		String html = null;
		Connection con = Jsoup.connect(url);
		con.validateTLSCertificates(false);

		con.timeout(30000);
		con.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
		con.header("Accept-Encoding", "gzip, deflate, sdch");
		con.header("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4");
		con.header("Cache-Control", "no-cache");
		con.header("Connection", "keep-alive");
		con.header("Pragma", "no-cache");
		con.header("Upgrade-Insecure-Requests", "1");
		con.header("User-Agent",
				"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36");
		
		Document doc = con.get();
		if (doc != null) {
			html = doc.html();
			PrintWriter w = new PrintWriter(filepath);
			w.println(html);
			w.flush();
			w.close();
		}
		
		//System.out.println(filepath);
		return html;
	}
	
	public static Document saveUrlToFileRDoc(String url, String filepath) throws Exception {
		Connection con = Jsoup.connect(url);
		con.validateTLSCertificates(false);

		con.timeout(30000);
		con.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
		con.header("Accept-Encoding", "gzip, deflate, sdch");
		con.header("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4");
		con.header("Cache-Control", "no-cache");
		con.header("Connection", "keep-alive");
		con.header("Pragma", "no-cache");
		con.header("Upgrade-Insecure-Requests", "1");
		con.header("User-Agent",
				"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36");
		
		Document doc = con.get();
		if (doc != null) {
			String html = doc.html();
			PrintWriter w = new PrintWriter(filepath);
			w.println(html);
			w.flush();
			w.close();
		}
		
		//System.out.println(filepath);
		return doc;
	}
	
	
	/**
	 *  如果url不含有http则给其拼接上http
	 *	如果url不含有https则给其拼接上https 
	 *	如果url不含有www.则给其拼接上www.
	 *	返回真实可访问的URL,否则返回空
	 */
	public static String saveSnapshot(String ym, String filepath){
		String result = null;
		List<String> results = getUrls(ym);
		for(String ur : results){
			try {
				saveUrlToFile(ur, filepath);
				result = ur;
				break;
			} catch (Exception e) {
			//	System.out.println(2);
				logger.error("获取网站快照异常:{}", e);
			}
		}
		return result;
	}
	
	public static boolean saveSnapshotb(String url, String filepath){
		boolean  result = false;
		List<String> results = getUrls(url);
		for(String ur : results){
			try {
				saveUrlToFile(ur, filepath);
				result = true;
				break;
			} catch (Exception e) {
				// TODO Auto-generated catch block
				logger.error("获取网站快照异常:{}", e);
			}
		}
		return result;
	}
	
	public static String saveSnapshotb_url(String url, String filepath){
		String  result = null;
		List<String> results = getUrls(url);
		for(String ur : results){
			try {
				String fp = saveUrlToFile(ur, filepath);
				if(fp!=null){
					result = ur;	
				}	
				break;
			} catch (Exception e) {
				e.printStackTrace();
				logger.error("获取快照异常", e);
			}
		}
		return result;
	}
	
	public static Document saveSnapshotb_url_doc(String url, String filepath){
		Document result = null;
		List<String> results = getUrls(url);
		for(String ur : results){
			try {
				Document fp = saveUrlToFileRDoc(ur, filepath);
				if(fp!=null){
					result = fp;
				}	
				break;
			} catch (Exception e) {
				// TODO Auto-generated catch block
				//System.out.println(result);
				//e.printStackTrace();
			}
		}
		return result;
	}
	
	public static List<String> getUrls(String url){
		List<String> lists = new ArrayList<String>();
		if(url == null || url.trim().equalsIgnoreCase(""))return lists;
		url = url.trim();
		if(!url.contains("http")){
			if(!url.contains("www.")){
				lists.add("http://www." + url);
				lists.add("http://" + url);
				lists.add("https://www." + url);
				lists.add("https://" + url);
			}else{
				lists.add("http://" + url);
				lists.add("https://" + url);
			}
		}else{
			lists.add(url);
		}
		return lists;
	}

	@Deprecated
	public static String saveUrlToEncodeFile(String url, String filepath) throws Exception {
		URL url1 = null;
		HttpURLConnection sconnection = null;
		InputStream is = null;
		OutputStream os = null;
		try {
			url1 = new URL(url);
			sconnection = (HttpURLConnection) url1.openConnection();
			if (url.startsWith("https")) {
				HttpsURLConnection hscon = (HttpsURLConnection) sconnection;
				hscon.setSSLSocketFactory(sslContext.getSocketFactory());
			}
			sconnection.setConnectTimeout(10000);
			sconnection.setReadTimeout(10000);
			sconnection.setRequestMethod("GET");
			sconnection.setDoOutput(true);

			sconnection.setRequestProperty("Accept",
					"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
			sconnection.setRequestProperty("Accept-Encoding", "gzip, deflate, sdch");
			sconnection.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4");
			sconnection.setRequestProperty("Cache-Control", "no-cache");
			sconnection.setRequestProperty("Connection", "keep-alive");
			sconnection.setRequestProperty("Pragma", "no-cache");
			sconnection.setRequestProperty("Upgrade-Insecure-Requests", "1");
			sconnection.setRequestProperty("User-Agent",
					"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.108 Safari/537.36");
			sconnection.connect();
			if (sconnection.getResponseCode() >= 400) {
				throw new IllegalStateException(url + "\t对方返回错误码:" + sconnection.getResponseCode());
			}
			String contentType = sconnection.getContentType();
			String cencoding = sconnection.getHeaderField("Content-Encoding");
			String encode = null;
			int ccindex = contentType.indexOf("charset=");
			if (ccindex >= 0) {
				encode = contentType.substring(ccindex + 8);
				int inx = encode.indexOf(";");
				if (inx > 0) {
					encode = encode.substring(0, inx).trim().toUpperCase();
				}
				if (encode.contains("ASCII"))
					encode = "UTF-8";
				else if (encode.contains("GBK") || encode.contains("GB2312") || encode.contains("ISO-8859-2")
						|| encode.contains("EUC-TW"))
					encode = "GB18030";
			} else {
				encode = "UNKNOW";
			}
			is = sconnection.getInputStream();
			InputStream gis = null;
			if (cencoding != null && cencoding.toLowerCase().contains("gzip")) {
				gis = new GZIPInputStream(is);
			} else {
				gis = is;
			}
			os = new FileOutputStream(filepath + "." + encode);
			int ch = -1;
			while ((ch = gis.read()) != -1) {
				os.write(ch);
			}
			return filepath + "." + encode;
		} catch (Exception e) {
			throw e;
		} finally {
			if (sconnection != null)
				sconnection.disconnect();
			if (is != null)
				is.close();
			if (os != null)
				os.close();
		}
	}
	/**
	 * 获取内网快照的 title。。。。
	 * @param nwUrl   内网快照地址
	 * */
	public static Map<String,Object>  getNwInfo(String nwUrl){
		Map<String,Object> map = new HashMap<String,Object>();
		String titleOld= "";
		String keywordsOld= "";
		String descriptionOld= "";
		File file = new File(nwUrl);
		Document doc;
		try {
			doc = Jsoup.parse(file,"UTF-8","");
		//	String html = doc.html();
			if(doc != null){
				
				if(doc.title() != null && !doc.title().equals("")){
					titleOld = doc.title();
				}else{
					titleOld = null;
				}
				//keywordsOld =  doc.select("meta[name=keywords]").attr("content");
				if(doc.select("meta[name=keywords]") != null){
					keywordsOld =  doc.select("meta[name=keywords]").attr("content");
				}else{
					keywordsOld = null;
				}
				
				if(doc.select("meta[name=description]") != null){
					descriptionOld =  doc.select("meta[name=description]").attr("content");
				}else{
					descriptionOld = null;
				}
			
				map.put("keywordsOld", keywordsOld);
				map.put("titleOld", titleOld);
				map.put("descriptionOld", descriptionOld);
				return map;
			}
		} catch (Exception e) {
			// TODO Auto-generated catch block
			logger.error("获取快照信息异常", e);
		}
	
		return null;
	}
	/**
	 * 获取外网快照的 title。。。。
	 * @param url 外网地址
	 * @throws Exception 
	 * */
	public static Map<String,Object>  getWwInfo(String url, String filePath) throws Exception{
		Map<String,Object> map = new HashMap<String,Object>();
		String titleNew= "";
		String keywordsNew= "";
		String descriptionNew= "";
		String html ="";
		
		Document doc = saveSnapshotb_url_doc(url, filePath);
		try {
			if(doc != null){
				html= doc.html();
				
				if(doc.title() != null && !doc.title().equals("")){
					titleNew = doc.title();
				}else{
					titleNew = null;
				}
				
				if(doc.select("meta[name=keywords]") != null){
					keywordsNew = doc.select("meta[name=keywords]").attr("content");
				}else{
					keywordsNew = null;
				}
				
				if(doc.select("meta[name=description]") != null){
					descriptionNew = doc.select("meta[name=description]").attr("content");
				}else{
					descriptionNew = null;
				}
				
				map.put("titleNew", titleNew);
				map.put("keywordsNew", keywordsNew);
				map.put("descriptionNew", descriptionNew);
				return map;
			}
		} catch (Exception e) {
			logger.error("获取外网快照信息异常", e);
		}
		
		return null;
		
	}
	
//	public static void main(String[] args) {
//		Map<String,Object> map = new HashMap<String,Object>();
//		String titleOld= "";
//		String keywordsOld= "";
//		String descriptionOld= "";
//		File file = new File("/datanfs/rzx/sync/snapshot/F5E29B986C6BDEF32CB8594C9694C600");
//		Document doc;
//		try {
//			doc = Jsoup.parse(file,"UTF-8","");
//			//String html = doc.html();
//			titleOld = doc.title();
//			keywordsOld =  doc.select("meta[name=keywords]").attr("content");
//			descriptionOld =  doc.select("meta[name=description]").attr("content");
//			System.out.println(titleOld);
//			System.out.println(keywordsOld);
//			System.out.println(descriptionOld);
//			map.put("keywordsOld", keywordsOld);
//			map.put("titleOld", titleOld);
//			map.put("descriptionOld", descriptionOld);
//		} catch (IOException e) {
//			// TODO Auto-generated catch block
//			e.printStackTrace();
//			
//		}
//	}
	
	//获取元搜索关键字
	public static List<String> querykeywords(String path){
		List<String> keywords = null;
		try {
			keywords = FileUtils.readLines(new File(path),"UTF-8");
			System.out.println(keywords);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return keywords;
	}
	
	

//	public static void main(String[] args) throws Exception {
////		downandcal(args);
////		 testsaveurl(args);
//		// testcalword(args);
//		 testSnap("C:\\Users\\xubin\\workspace\\funds_task\\parsnum", 100);
//	}
//	
//	private static int num;
//	
//	public static synchronized void addNum(){
//		num ++;
//		System.out.println(num);
//	}
//	
//	public static void testSnap(String file, int threadNum) throws IOException{
//		ExecutorService send = Executors.newFixedThreadPool(threadNum);
//		List<String> results = FileUtils.readLines(new File(file), "utf-8");
//		for(final String domain : results){
//			Thread thread = new Thread(){
//				@Override
//				public void run() {
//					// TODO Auto-generated method stub
//					if(!saveSnapshotb(domain, "d:/html")){
//						System.out.println(domain);
//					}
//					addNum();
//				}
//			};
//			send.execute(thread);
//		}
//	}


//	private static void downandcal(String[] args) throws Exception {
//
//		ExecutorService send = Executors.newFixedThreadPool(200);
//		if (args.length <= 1)
//			System.out.println("参数错误!");
//		String type = args[0];
//		if ("1".equals(type)) {// 下载url，输出到文件
//			final String urlfile = args[1];
//			final String savepath = args[2];
//			BufferedReader reader = new BufferedReader(new FileReader(urlfile));
//			try {
//				while (true) {
//					String fline = reader.readLine();
//					if (fline == null)
//						break;
//					if (fline.trim().length() == 0)
//						continue;
//					final String url = "http://" + fline.trim();
//					try {
//						send.execute(new Runnable() {
//							@Override
//							public void run() {
//								try {
//									saveUrlToFile(url.trim(), savepath + getDomainFromUrl(url));
//								} catch (Exception e) {
//									System.out.println(url);
//								}
//							}
//						});
//					} catch (Exception e) {
//						// e.printStackTrace();
//						continue;
//					}
//				}
//			} catch (Exception e) {
//				// e.printStackTrace();
//			} finally {
//				reader.close();
//			}
//			Thread.sleep(10000);
//			send.shutdown();
//			while (true) {
//				if (send.isTerminated()) {
//					System.out.println("下载url结束了!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
//					break;
//				}
//				Thread.sleep(200);
//			}
//		} else if ("2".equals(type)) {// 验证独立词，输出到文件
//			final HashMap<String, Integer> keywords = getJRKeywords();
//			final String urlfile = args[1];
//			final String savepath = args[2];
//			final String restfile = args[3];
//			BufferedReader reader = new BufferedReader(new FileReader(urlfile));
//			final PrintWriter writer = new PrintWriter(restfile);
//			try {
//				while (true) {
//					String fline = reader.readLine();
//					if (fline == null)
//						break;
//					fline = fline.trim();
//					String url = "http://" + fline.trim();
//					final String domain = getDomainFromUrl(url);
//					final String htmlfile = savepath + domain;
//					try {
//						send.execute(new Runnable() {
//							@Override
//							public void run() {
//								try {
//									int value = DocumentUtil.calcWordCapInHtmlFile(keywords, htmlfile);
//									if (value != -1)
//										writeFile(writer, domain + "\t" + value);
//								} catch (Exception e) {
//									e.printStackTrace();
//								}
//							}
//						});
//					} catch (
//
//					Exception e)
//
//					{
//						// e.printStackTrace();
//						continue;
//					}
//				}
//			} catch (Exception e) {
//				// e.printStackTrace();
//			} finally {
//				reader.close();
//			}
//			Thread.sleep(10000);
//			send.shutdown();
//			while (true)
//
//			{
//				if (send.isTerminated()) {
//					System.out.println("计算独立词结束了!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
//					writer.close();
//					break;
//				}
//				Thread.sleep(200);
//			}
//
//		} else if ("3".equals(type)) {// 提取排除词
//			final String urlfile = args[1];
//			final String savepath = args[2];
//			final String restfile = args[3];
//			BufferedReader reader = new BufferedReader(new FileReader(urlfile));
//			final PrintWriter writer = new PrintWriter(restfile);
//			try {
//				while (true) {
//					String fline = reader.readLine();
//					if (fline == null)
//						break;
//					fline = fline.trim();
//					String url = "http://" + fline.trim();
//					final String domain = getDomainFromUrl(url);
//					final String htmlfile = savepath + domain;
//					try {
//						send.execute(new Runnable() {
//							@Override
//							public void run() {
//								try {
//									HashMap<String, Integer> keymap = DocumentUtil
//											.getIsolatedWordFromHtmlFileExBody(htmlfile);
//									if (keymap == null)
//										return;
//									Set<String> keyset = keymap.keySet();
//									for (String key : keyset) {
//										if (key.length() > 3 && key.length() < 9)
//											writeFile(writer, key);
//									}
//
//								} catch (Exception e) {
//									e.printStackTrace();
//								}
//							}
//						});
//					} catch (Exception e) {
//						// e.printStackTrace();
//						continue;
//					}
//				}
//			} catch (Exception e) {
//				// e.printStackTrace();
//			} finally {
//				reader.close();
//			}
//			Thread.sleep(10000);
//			send.shutdown();
//			while (true) {
//				if (send.isTerminated()) {
//					System.out.println("提取排除词结束了");
//					writer.close();
//					break;
//				}
//				Thread.sleep(200);
//			}
//
//		} else if ("4".equals(type)) {// 排除筛选
//			final String urlfile = args[1];
//			final String savepath = args[2];
//			final String restfile = args[3];
//			BufferedReader reader = new BufferedReader(new FileReader(urlfile));
//			final PrintWriter writer = new PrintWriter(restfile);
//			final String[] keywords = getNotJRKeywords();
//			try {
//				while (true) {
//					String fline = reader.readLine();
//					if (fline == null)
//						break;
//					fline = fline.trim();
//					String url = "http://" + fline.trim();
//					final String domain = getDomainFromUrl(url);
//					final String htmlfile = savepath + domain;
//					try {
//						send.execute(new Runnable() {
//							@Override
//							public void run() {
//								try {
//									int value = DocumentUtil.isIncludeWords(keywords, htmlfile);
//									writeFile(writer, domain + "\t" + value);
//								} catch (Exception e) {
//									e.printStackTrace();
//								}
//							}
//						});
//					} catch (Exception e) {
//						// e.printStackTrace();
//						continue;
//					}
//				}
//			} catch (Exception e) {
//				// e.printStackTrace();
//			} finally {
//				reader.close();
//			}
//			Thread.sleep(10000);
//			send.shutdown();
//			while (true) {
//				if (send.isTerminated()) {
//					System.out.println("排除词结束了");
//					writer.close();
//					break;
//				}
//				Thread.sleep(200);
//			}
//
//		}
//	}
	
	public static String ip138(String ip){
		String target = null;
		try{
			String url = "http://www.ip138.com/ips138.asp?ip=" + ip + "&action=2";
			Document doc = HttpUtil.getWebDocument(url);
			Elements eles = doc.getElementsByClass("ul1");
			if(eles != null && eles.size() > 0){
				Element ele = eles.get(0);
				Elements childrens = ele.children();
				if(childrens != null && childrens.size() > 0){
					for(int i = 0 ; i < childrens.size() ; i ++){
						Element chele = childrens.get(i);
						String text = chele.ownText();
						if(text != null && text.contains("本站数据：")){
							target = text.replaceAll("本站数据：", "");
						}
					}
				}
			}
		}catch(Exception e){
			e.printStackTrace();
			logger.error("通过IP获取地市信息异常", e);
		}
		return target;
	}

	public static Response getResponse(String url){
		if(!StringUtils.isEmpty(url)){
			try {
				Response response = Jsoup.connect(url).validateTLSCertificates(false)
						.userAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1042.0 Safari/535.21")
						.timeout(30000).header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
						.header("Accept-Encoding", "gzip, deflate, sdch").header("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4")
						.header("Connection", "keep-alive").header("Cache-Control", "no-cache").header("Pragma", "no-cache")
						.header("Upgrade-Insecure-Requests", "1")
						.execute();
				return response;
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
				logger.error("jsoup打开网站异常", e);
			}
			return null;
		}
		throw new RuntimeException("Url不能为空或者null。。");
	}
	
	public static Response getResponse(String url, int timeout){
		if(!StringUtils.isEmpty(url)){
			try {
				Response response = Jsoup.connect(url).validateTLSCertificates(false)
						.userAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1042.0 Safari/535.21")
						.timeout(timeout).header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
						.header("Accept-Encoding", "gzip, deflate, sdch").header("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4")
						.header("Connection", "keep-alive").header("Cache-Control", "no-cache").header("Pragma", "no-cache")
						.header("Upgrade-Insecure-Requests", "1")
						.execute();
				return response;
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
				logger.error("jsoup打开网站:{}异常, 异常信息:{}", url, e.getMessage());
			}
			return null;
		}
		throw new RuntimeException("Url不能为空或者null。。");
	}
	
	public static Document getCCWebDocument(String url){
		Response response = getResponse(url);
		if(response == null)return null;
		
		try {
			
			Document parse = response.parse();
//			URL url2 = response.url();   //重定向之后的url
			return parse;
			
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			logger.error("jsoup解析异常", e);
			
		}
		return null;
	}
	
	
	
	public static Document getWebDocument(String url) {
		Document doc = null;
		Connection con = null;
		try{
			con = Jsoup.connect(url);
			con.validateTLSCertificates(false);

			con.timeout(30000);
			con.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
			con.header("Accept-Encoding", "gzip, deflate, sdch");
			con.header("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4");
			con.header("Cache-Control", "no-cache");
			con.header("Connection", "keep-alive");
			con.header("Pragma", "no-cache");
			con.header("Upgrade-Insecure-Requests", "1");
			con.header("User-Agent",
					"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36");
			
			doc = con.get();
//			con.execute().parse()
//			
//			Response response = Jsoup.connect(url).validateTLSCertificates(true)
//					.userAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1042.0 Safari/535.21")
//					.timeout(10000)
//					.execute();
//			URL urljump = response.url();
			
			
		}catch(Exception e){
			e.printStackTrace();
			logger.error("jsoup获取网站Docment异常", e);
		}
		return doc;
	}
	
	public static String getDocmentByYm(String ym){
		String  result = null;
		List<String> results = getUrls(ym);
		for(String url : results){
			try {
				Document fp = getWebDocument(url);
				if(fp != null){
					result = url;
					break;
				}	
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		return result;
	}
	
//	public static List<Object> getObjects(Document doc, String xpath){
//		List<Object> target = null;
//		if(doc == null)return target;
//		try{
////			System.out.println(doc);
//			JXDocument jxDocument = new JXDocument(doc);
//			List<Object> rs = jxDocument.sel(xpath);
//			target = rs;
//		}catch(Exception e){
//			e.printStackTrace();
//		}
//		return target;
//	}
	
	public static List<Element> oprateObjects(List<Object> objs){
		List<Element> target = new ArrayList<Element>();
		if(objs != null){
			for(Object obj : objs){
				if(obj != null && obj instanceof Element){
					target.add((Element)obj);
				}
			}
		}
		return target;
	}

	public static String getDomainFromUrl(String url) {
		url = url.trim();
		if (url.startsWith("https://"))
			url = url.substring(8);
		else if (url.startsWith("http://"))
			url = url.substring(7);
		if (url.startsWith("www."))
			url = url.substring(4);
		int end = url.indexOf("/");
		if (end > 0)
			url = url.substring(0, end);
		end = url.indexOf("?");
		if (end > 0)
			url = url.substring(0, end);
		return url;
	}
	
	public static String getDomainByUrl(String url){
		String domain = null;
		try{
			String wym = StringParseUtils.parseString(url, "([^\\u4e00-\\u9fa5 ^\\. ^/ ^\\s ^.]+(\\.)){1,}[a-z]+", false, 0);
			if(wym != null){
				wym = wym.replaceAll("www\\.", "");
				domain = wym;
			}
		}catch(Exception e){
			e.printStackTrace();
		}
		return domain;
	}
	
	public static String getDomainByUrl(String url, int num){
		String domain = null;
		try{
			String wym = StringParseUtils.parseString(url, "([^\\u4e00-\\u9fa5 ^\\. ^/ ^\\s ^.]+(\\.)){1,}[a-z]+", num);
			if(wym != null){
				wym = wym.replaceAll("www\\.", "");
				domain = wym;
			}
		}catch(Exception e){
			e.printStackTrace();
		}
		return domain;
	}
	
	public static String getDomainName(String url)
    {
		try {
			String host = new URL(url).getHost().toLowerCase();// 
			//System.out.println(host);
			Pattern pattern = Pattern.compile("[^\\.]+(\\.com\\.cn|\\.net\\.cn|\\.org\\.cn|\\.gov\\.cn|\\.com|\\.net|\\.cn|\\.org|\\.cc|\\.me|\\.tel|\\.mobi|\\.asia|\\.biz|\\.info|\\.name|\\.tv|\\.hk|\\.公司|\\.中国|\\.网络)");
			Matcher matcher = pattern.matcher(host);
			while (matcher.find()) {
				return matcher.group();
			}
			
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return  null;
	}
	
	public static String getUrl(String urlStr){
		String url = null;
		try{
			String wym = StringParseUtils.parseString(urlStr, "(http(s)?://)?([^\\u4e00-\\u9fa5 ^\\. ^/ ^\\s ^: ^：]*(\\.)){1,}[a-z]+", false, 0);
			url = wym;
			if(url != null && !url.trim().equalsIgnoreCase("")){
				if(!url.contains("http")){
					url = "http://" + url;
				}
			}
		}catch(Exception e){
			e.printStackTrace();
		}
		return url;
	}
	
	public static String getUrl(String urlStr, int num){
		String url = null;
		try{
			if(urlStr == null || urlStr.trim().equalsIgnoreCase(""))return null;
			String wym = StringParseUtils.parseString(urlStr, "(http(s)?://)?([^\\u4e00-\\u9fa5 ^\\. ^/ ^\\s]*(\\.)){1,}[a-z]+", num);
			url = wym;
			if(url != null){
				if(!url.contains("http")){
					url = "http://" + url;
				}
			}
		}catch(Exception e){
			e.printStackTrace();
		}
		return url;
	}
	
	public static void download(String urlString, String filename) throws Exception {
	     // 构造URL
	     URL url = new URL(urlString);
	     // 打开连接
	    URLConnection con = url.openConnection();
	     // 输入流
	    InputStream is = con.getInputStream();
	     // 1K的数据缓冲
	    byte[] bs = new byte[1024];
	     // 读取到的数据长度
	    int len;
	     // 输出的文件流
	    OutputStream os = new FileOutputStream(filename);
	     // 开始读取
	    while ((len = is.read(bs)) != -1) {
	       os.write(bs, 0, len);
	     }
	     // 完毕，关闭所有链接
	    os.close();
	    is.close();
	 }
	
	public static String getDomainShortNameByEnd(String domain) {
		String efix = null;
		for (String end : topd) {
			if (domain.endsWith(end)) {
				domain = domain.substring(0, domain.length() - end.length());
				efix = end;
				break;
			}
		}
		if (efix == null) {
			int index = domain.lastIndexOf(".");
			if (index > 0) {
				efix = domain.substring(index);
				domain = domain.substring(0, index);
			} else
				return domain;
		}
		String shortName = domain;
		int index = domain.lastIndexOf(".");
		if (index >= 0) {
			shortName = domain.substring(index + 1);
		}
		return shortName + efix;

	}
	
	public static HtmlPage getHtmlPageByDomain(String ym){
		HtmlPage page = null;
		List<String> urls = getUrls(ym);
		for(String url : urls){
			page = getHtmlPageByUrl(url);
			if(page != null)break;
		}
		return page;
	}
	
	public static String getUrlByHtmlPageDomain(String ym){
		String result = null;
		List<String> urls = getUrls(ym);
		for(String url : urls){
			HtmlPage page = getHtmlPageByUrl(url);
			if(page != null){
				result = url;
				break;
			}
		}
		return result;
	}
	
	public static HtmlPage getHtmlPageByUrl(String url){
		HtmlPage page = null;
		WebClient webClient = null;
		try{
			webClient = new WebClient();
			webClient.getOptions().setJavaScriptEnabled(false);
	        webClient.getOptions().setCssEnabled(false);
	        webClient.getOptions().setActiveXNative(false);
	        webClient.getOptions().setThrowExceptionOnScriptError(false);
	        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
	        webClient.getOptions().setUseInsecureSSL(true);
	        webClient.getOptions().setTimeout(10 * 1000);
	        page = webClient.getPage(url);
		}catch(Exception e){
			e.printStackTrace();
		}finally{
//			if(webClient != null)webClient.close();
		}
		return page;
	}
	
	public static String sendGetRequest(String reqURL, String decodeCharset){  
        long responseLength = 0;       //响应长度  
        String responseContent = null; //响应内容  
        HttpClient httpClient = new DefaultHttpClient(); //创建默认的httpClient实例
        HttpGet httpGet = new HttpGet(reqURL);           //创建org.apache.http.client.methods.HttpGet
        try{  
            HttpResponse response = httpClient.execute(httpGet); //执行GET请求
            HttpEntity entity = response.getEntity();            //获取响应实体
            if(null != entity){  
                responseLength = entity.getContentLength();  
                responseContent = EntityUtils.toString(entity, decodeCharset==null ? "UTF-8" : decodeCharset);
                EntityUtils.consume(entity); //Consume response content
            }  
            System.out.println("请求地址: " + httpGet.getURI());  
            System.out.println("响应状态: " + response.getStatusLine());  
            System.out.println("响应长度: " + responseLength);  
            System.out.println("响应内容: " + responseContent);  
        }catch(ClientProtocolException e){
            logger.debug("该异常通常是协议错误导致,比如构造HttpGet对象时传入的协议不对(将'http'写成'htp')或者服务器端返回的内容不符合HTTP协议要求等,堆栈信息如下", e);  
        }catch(ParseException e){
            logger.debug(e.getMessage(), e);  
        }catch(IOException e){  
            logger.debug("该异常通常是网络原因引起的,如HTTP服务器未启动等,堆栈信息如下", e);  
        }finally{  
            httpClient.getConnectionManager().shutdown(); //关闭连接,释放资源  
        }  
        return responseContent;
    }
	
	public static PhantomJSDriver getPhantomJSDriver(){
		PhantomJSDriver target = null;
		DesiredCapabilities dcp = new DesiredCapabilities();
		dcp.setJavascriptEnabled(true);
		dcp.setPlatform(Platform.LINUX);
		dcp.setBrowserName("Chrome");
		dcp.setCapability("Accept-Language", "zh-CN,zh;q=0.8");
		dcp.setCapability("phantomjs.page.settings.resourceTimeout", 5000);
		dcp.setCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36");
		dcp.setCapability(PhantomJSDriverService.PHANTOMJS_EXECUTABLE_PATH_PROPERTY, phantomjsPath);
		String [] phantomJsArgs = {"--ignore-ssl-errors=true", "--ssl-protocol=any", "--output-encoding=utf-8", "--script-encoding=utf-8"};
		dcp.setCapability(PhantomJSDriverService.PHANTOMJS_CLI_ARGS, phantomJsArgs);
		target = new PhantomJSDriver(dcp);
		target.manage().window().setSize(new Dimension(1440, 900));
		return target;
	}
	
	/**
	 * 
	 * @param plat
	 * @param browserName
	 * @param javasriptenabled
	 * @param phantomjsPath
	 * @param capabilitys
	 * @param phantomJsArgs
	 * @param dimensionW
	 * @param dimensionH
	 * @return
	 */
	public static PhantomJSDriver getPhantomJSDriver(Platform plat, String browserName, boolean javasriptenabled, String phantomjsPath, Map<String, Object> capabilitys, String[] phantomJsArgs, Integer dimensionW, Integer dimensionH){
		PhantomJSDriver target = null;
		DesiredCapabilities dcp = new DesiredCapabilities();
		dcp.setJavascriptEnabled(javasriptenabled);
		dcp.setPlatform(plat);
		dcp.setBrowserName(browserName);
		if(capabilitys != null && !capabilitys.isEmpty()){
			for(String key : capabilitys.keySet()){
				dcp.setCapability(key, capabilitys.get(key));
			}
		}
		if(dimensionW == null || dimensionH == null || dimensionW == 0 || dimensionH == 0){
			dimensionW = 1440;
			dimensionH = 900;
		}
		dcp.setCapability(PhantomJSDriverService.PHANTOMJS_CLI_ARGS, phantomJsArgs);
		target = new PhantomJSDriver(dcp);
		target.manage().window().setSize(new Dimension(dimensionW, dimensionH));
		return target;
	}
	
	public static String getPhantomJSHtmlByUrl(String url){
		String html = null;
		PhantomJSDriver driver = null;
		try{
			driver = getPhantomJSDriver();
			driver.get(url);
			html = driver.getPageSource();
			Thread.sleep(5000);
			WebElement ele = driver.findElementByCssSelector("body > div:nth-child(3) > div > div > div.left_part > div > div:nth-child(1) > h2 > a");
			if(ele != null){
				System.out.println("发现该元素");
			}
		}catch(Exception e){
			e.printStackTrace();
		}finally{
			if(driver != null){
				driver.close();
				driver.quit();
			}
		}
		return html;
	}
	
	/**
	 * CSS单网页定向采集
	 * @param url
	 * @param detail
	 * @param openSleep
	 * @return
	 */
	public static Map<String, String> directionalCollectByCss(String url, Map<String, String> detail, long openSleep){
		 Map<String, String> result = new HashMap<String, String>();
		 
		 return result;
	}
	
	public static void pressCtrlF(PhantomJSDriver driver){
		try{
			System.out.println(Keys.LEFT_CONTROL);
			driver.getKeyboard().pressKey(Keys.LEFT_CONTROL);
			Thread.sleep(100);
			driver.getKeyboard().pressKey("f");
			Thread.sleep(100);
			driver.getKeyboard().releaseKey("f");
			Thread.sleep(100);
			driver.getKeyboard().releaseKey(Keys.LEFT_CONTROL);
		}catch(Exception e){
			e.printStackTrace();
		}
	}
	
	public static void pressEnter(PhantomJSDriver driver){
		try{
			System.out.println(Keys.ENTER);
			driver.getKeyboard().pressKey(Keys.ENTER);
			Thread.sleep(100);
			driver.getKeyboard().releaseKey(Keys.ENTER);
		}catch(Exception e){
			e.printStackTrace();
		}
	}
	
	public static boolean writePictureByWebUrl(String url, String destFileDir, String fileName, String keyword){
		boolean target = false;
		PhantomJSDriver driver = null;
		try{
			driver = getPhantomJSDriver();
			driver.navigate().to(url);
			Thread.sleep(2000);
			if(keyword != null && !keyword.trim().equalsIgnoreCase("")){
				driver.executeScript("window.find('" + keyword + "',false,false);", "");				
			}
			byte[] images = driver.getScreenshotAs(OutputType.BYTES);
			File destfile = new File(destFileDir);
			if(!destfile.exists())destfile.mkdirs();
			File rdestFile = new File(destFileDir + "/" + fileName);
			if(rdestFile.exists())rdestFile.delete();
			FileUtils.writeByteArrayToFile(rdestFile, images);
			if(rdestFile.exists())target = true;
		}catch(Exception e){
			logger.error("获取网页图片异常", e);
		}finally{
			if(driver != null){
				try{
					driver.close();
				}catch(Exception e){
					e.printStackTrace();
				}
				try{
					driver.quit();
				}catch(Exception e){
					e.printStackTrace();
				}
			}
		}
		return target;
	}

	
	/**
     * fileName 文件名(不带后缀)
     * filePath 文件的本地路径 (xxx / xx / test.jpg)
     */ 
    public static void uploadFile(String fileName, String filePath, String host) {

        HttpURLConnection conn = null;

        /// boundary就是request头和上传文件内容的分隔符(可自定义任意一组字符串)
        String BOUNDARY = "******";
        // 用来标识payLoad+文件流的起始位置和终止位置(相当于一个协议,告诉你从哪开始,从哪结束)
        String  preFix = ("\r\n--" + BOUNDARY + "--\r\n");

        try {
            // (HttpConst.uploadImage 上传到服务器的地址
            URL url = new URL(host);
            conn = (HttpURLConnection) url.openConnection();
            conn.setConnectTimeout(5000);
            conn.setReadTimeout(30000);
            conn.setDoOutput(true);
            conn.setDoInput(true);
            conn.setUseCaches(false);
            // 设置请求方法
            conn.setRequestMethod("POST");
            // 设置header
            conn.setRequestProperty("Accept","*/*");
            conn.setRequestProperty("Connection", "keep-alive");
            conn.setRequestProperty("Content-Type",
                    "multipart/form-data; boundary=" + BOUNDARY);
            // 获取写输入流
            OutputStream out = new DataOutputStream(conn.getOutputStream());
            // 获取上传文件
            File file = new File(filePath);

            // 要上传的数据
            StringBuffer strBuf = new StringBuffer();

            // 标识payLoad + 文件流的起始位置
            strBuf.append(preFix);

            // 下面这三行代码,用来标识服务器表单接收文件的name和filename的格式
            // 在这里,我们是file和filename.后缀[后缀是必须的]。
            // 这里的fileName必须加个.jpg,因为后台会判断这个东西。
            // 这里的Content-Type的类型,必须与fileName的后缀一致。
            // 如果不太明白,可以问一下后台同事,反正这里的name和fileName得与后台协定！
            // 这里只要把.jpg改成.txt，把Content-Type改成上传文本的类型，就能上传txt文件了。
            strBuf.append("Content-Disposition: form-data; name=\"file\"; filename=\"" + fileName+".jpg" + "\"\r\n");
            strBuf.append("Content-Type: image/jpeg"  + "\r\n\r\n");
            out.write(strBuf.toString().getBytes());

            // 获取文件流
            FileInputStream fileInputStream = new FileInputStream(file);
            DataInputStream inputStream = new DataInputStream(fileInputStream);

            // 每次上传文件的大小(文件会被拆成几份上传)
            int bytes = 0;
            // 计算上传进度
            float count = 0;
            // 获取文件总大小
            int fileSize = fileInputStream.available();
            // 每次上传的大小
            byte[] bufferOut = new byte[1024];
            // 上传文件
            while ((bytes = inputStream.read(bufferOut)) != -1) {
                // 上传文件(一份)
                out.write(bufferOut, 0, bytes);
                // 计算当前已上传的大小
                count += bytes;
                // 打印上传文件进度(已上传除以总大*100就是进度)
                logger.info("progress:" +(count / fileSize * 100) +"%");
            }

            // 关闭文件流
            inputStream.close();

            // 标识payLoad + 文件流的结尾位置
            out.write(preFix.getBytes());

            // 至此上传代码完毕

            // 总结上传数据的流程：preFix + payLoad(标识服务器表单接收文件的格式) + 文件(以流的形式) + preFix
            // 文本与图片的不同,仅仅只在payLoad那一处的后缀的不同而已。

            // 输出所有数据到服务器
            out.flush();

            // 关闭网络输出流
            out.close();

            // 重新构造一个StringBuffer,用来存放从服务器获取到的数据
            strBuf = new StringBuffer();

            // 打开输入流 , 读取服务器返回的数据
            BufferedReader reader = new BufferedReader(new
                    InputStreamReader(conn.getInputStream()));

            String line;

            // 一行一行的读取服务器返回的数据
            while ((line = reader.readLine()) != null) {
                strBuf.append(line).append("\n");
            }

            // 关闭输入流
            reader.close();

            // 打印服务器返回的数据
            logger.info("上传成功:"+strBuf.toString());

        } catch (Exception e) {
        	logger.info("上传图片出错:"+e.toString());
        } finally {
            if (conn != null) {
                conn.disconnect();
            }
        }

    }
    
    public static String getChecksum(String filename, String algorithm) throws Exception {
        byte[] b = createChecksum(filename, algorithm);
        String result = "";
        for (int i=0; i < b.length; i++) {
            result += Integer.toString( ( b[i] & 0xff ) + 0x100, 16).substring( 1 );
        }
        return result;
    }
    
	public static String getChecksum(byte[] buffer, String algorithm) throws Exception {
        byte[] b = createChecksum(buffer, algorithm);
        String result = "";
        for (int i=0; i < b.length; i++) {
            result += Integer.toString( ( b[i] & 0xff ) + 0x100, 16).substring( 1 );
        }
        return result;
    }
    
    public static byte[] createChecksum(String filename, String algorithm) throws Exception {
        InputStream fis =  new FileInputStream(filename);
        byte[] buffer = new byte[1024];
        MessageDigest complete = MessageDigest.getInstance(algorithm);
        int numRead;
        do {
            numRead = fis.read(buffer);
            if (numRead > 0) {
                complete.update(buffer, 0, numRead);
            }
        } while (numRead != -1);
        fis.close();
        return complete.digest();
    }
    
    public static byte[] createChecksum(byte[] buffer, String algorithm) throws Exception {
        MessageDigest complete = MessageDigest.getInstance(algorithm);
        complete.update(buffer, 0, buffer.length);
        return complete.digest();
    }
	
	public static String clawerSyncJsoupPost(String op_type, String taskId){
		String result = null;
		try{
			Connection con = Jsoup.connect(crawlerSyncUrl);
			con.validateTLSCertificates(false);
			con.timeout(30000);
			con.ignoreContentType(true);
			con.data("op_type", op_type);
			con.data("task_id", taskId);
			Document doc = con.post();
			result = doc.body().text();
		}catch(Exception e){
			logger.error("同步任务数据异常：{}", taskId, e);
		}
		return result;
	}
	
	
	public static void downloadPictureByUrl(String urlStr, String destDir, String fileName, String format){
		try {  
			URL url = new URL(urlStr);  
            DataInputStream dataInputStream = new DataInputStream(url.openStream());  
            String imageName = destDir + fileName + format;
            FileOutputStream fileOutputStream = new FileOutputStream(new File(imageName));  

            byte[] buffer = new byte[1024];  
            int length;  

            while ((length = dataInputStream.read(buffer)) > 0) {  
                fileOutputStream.write(buffer, 0, length);  
            }  

            dataInputStream.close();  
            fileOutputStream.close();  
        } catch (MalformedURLException e) {  
            e.printStackTrace();  
        } catch (IOException e) {  
            e.printStackTrace();  
        } 
	}
	
	public static void downloadMIAPPLogo(String appfxdz, String destDir, String fileName, String format){
		Document doc = HttpUtil.getWebDocument(appfxdz);
		Elements eles = doc.getElementsByClass("yellow-flower");
		if(eles.size() > 0){
			String urlStr = eles.get(0).attr("src");
			System.out.println(urlStr);
			downloadPictureByUrl(urlStr, destDir, fileName, format);
		}
	}
	
	public static void downloadBAIDUAPPLogo(String appfxdz, String destDir, String fileName, String format){
		Document doc = HttpUtil.getWebDocument(appfxdz);
		Elements eles = doc.getElementsByClass("app-pic");
		if(eles.size() > 0){
			String url = eles.get(0).getElementsByIndexEquals(0).attr("src");
			System.out.println(url);
			downloadPictureByUrl(url, destDir, fileName, format);
		}
	}
	
	public static void download360APPLogo(String appfxdz, String destDir, String fileName, String format){
		Document doc = HttpUtil.getWebDocument(appfxdz);
		Elements eles = doc.getElementsByClass("clearfix");
		if(eles.size() > 0){
			String url = eles.get(0).getElementsByIndexEquals(0).attr("src");
			System.out.println(url);
			downloadPictureByUrl(url, destDir, fileName, format);
		}
	}
	
	public static void downloadQQAPPLogo(String appfxdz, String destDir, String fileName, String format){
		Document doc = HttpUtil.getWebDocument(appfxdz);
		Elements eles = doc.getElementsByClass("det-icon");
		if(eles.size() > 0){
			String url = eles.get(0).getElementsByIndexEquals(0).attr("src");
			System.out.println(url);
			downloadPictureByUrl(url, destDir, fileName, format);
		}
	}
	
	
	public static void main(String[] args) throws Exception {
//		String ip = "125.211.197.147";
//		String values = ip138(ip);
//		System.out.println(values);
//		String url = "http://www.bbs.cnyingsbao.com:80/homeProjectListController/ordinary_info?YSPROJECT_ID=154719e10ca14414acbcb25c80fd90c7";
//		System.out.println(getDomainByUrl(url));
//		System.out.println(getDomainShortNameByEnd(getDomainByUrl(url)));

//		Map<String, Object> ts = HttpUtil.getNwInfo("C:\\Users\\xubin\\workspace\\funds_task\\file\\E6BFF102E97137F03AE37BE29D0402D4");
//		System.out.println(ts);
		
//		System.out.println(getCCWebDocument("http://www.ebaohulu.com"));

//		HtmlPage page = HttpUtil.getHtmlPageByUrl("http://www.guojinbao.com/aboutusCenter.shtml?channel=20");
//		System.out.println(page.asXml());
		
		//jded.cn
		//hsbc.com.hk
		//furuntouzi.com
		//kunluntrust.com
//		String ym = "http://www.minshenglife.com/templet/default/Notice.jsp?id=729&pid=728";
//		String result = HttpUtil.getDocmentByYm(ym);
//		System.out.println(result);
//		result = HttpUtil.getUrlByHtmlPageDomain(ym);
//		System.out.println(result);
//		System.out.println(ts);
		//http://www.hongyuanqh.com
//		String url = "http://chinarsol.com";
//		HtmlPage page = HttpUtil.getHtmlPageByUrl(url);
//		System.out.println(page.asXml());
//		System.out.println(page.asXml().length());
//		Document doc = HttpUtil.getWebDocument(url);
//		System.out.println(doc.html());
//		System.out.println(doc.getAllElements().size());
//		
//		String str = doc.toString();
//		System.out.println(StringParseUtils.parseString(str, ".*\\.?location(\\.[a-z]*)?\\s*=.*", false));
//		String html = FileUtils.readFileToString(new File("C:\\Users\\xubin\\workspace\\funds_task\\file\\E6BFF102E97137F03AE37BE29D0402D4"), "utf-8");
//		String title = getHtmlTitle(html);
//		String description = getHtmlDescription(html);
//		String bodyText = getHtmlBodyText(html);
//		System.out.println("Title :" + title);
//		System.out.println("Description :" + description);
//		System.out.println(StringParseUtils.parseString(html, "<\\s*body\\s*[^<^>]*>([\\s\\S]*)</\\s*body\\s*[^<^>]*>", true));
//		System.out.println("BodyText: " + bodyText);
//		String url = "http://s1.qhdnews.com";
//		Response res = HttpUtil.getResponse(url);
//		System.out.println(getUrlByHtmlPageDomain("trjcn.org"));
//		String result = clawerSyncJsoupPost("download", "1455496");
//		System.out.println(result);
//		String url = "https://www.zt.com/News/index/cat_key/daikuan_notice.html";
//		String filePath = "D:/gsxx";
//		String fileName = "t330763.jpg";
//		String keyword = null;
//		String html = getPhantomJSHtmlByUrl(url);
		String url1 = "http://127.0.0.1:80";
//		String url2 = "http://www.ppmoney.com";
		PhantomJSDriver driver = getPhantomJSDriver();
		driver.navigate().to(url1);
		System.out.println(driver.getPageSource());
		driver.quit();
//		System.out.println(driver.getWindowHandles().size());
//		driver.get(url1);
//		System.out.println(driver.getWindowHandles().size());
//		driver.get(url2);
//		System.out.println(driver.getWindowHandles().size());
//		System.out.println(html);
//		Document doc = Jsoup.parse(html);
//		cleanUpHidinTags(doc);
//		String shtml = doc.html();
//		shtml = HttpUtil.clearHtmlNotes(shtml);
//		System.out.println(shtml);
//		HttpUtil.writePictureByWebUrl(url, filePath, fileName, keyword);
//		System.out.println(writePictureByWebUrl(url, filePath, fileName));
//		String host = "http://202.108.211.56:13121/onload";
//		uploadFile(fileName, filePath, host);
//		System.out.println(getChecksum(filePath + "/" + fileName, "SHA"));
//		String file_path = "C:/Users/xubin/workspace/funds_task/file/FDD5D52355A301F606027F3DF53BE3FA";
//		Document doc = Jsoup.parse(new File(file_path), "utf-8");
//		cleanUpHidinTags(doc);
//		String shtml = doc.html();
//		shtml = HttpUtil.clearHtmlNotes(shtml);
//		System.out.println(shtml);
//		String filePath = "D:/gsxx/qqapp.csv";
//		String encoding = "utf-8";
//		String dir = "D:/gsxx/qq/";
//		String format = ".jpg";
//		List<String> urls = FileUtils.readLines(new File(filePath), encoding);
//		for(String turl : urls){
//			String[] asg = turl.split(",");
//			String name = asg[0];
//			String url = asg[1];
//			downloadQQAPPLogo(url, dir, name, format);
//		}
//		System.out.println(HttpUtil.getDomainByUrl(url1));
//		Response res = HttpUtil.getResponse("http://www.my089.com");
//		System.out.println(res.url().toString());
//		System.out.println(HttpUtil.getDomainByUrl(url1, 1));
	}
	
}
